package com.study.bigdata.spark.streaming

import org.apache.kafka.clients.consumer.{ConsumerConfig, ConsumerRecord}
import org.apache.spark.SparkConf
import org.apache.spark.streaming.dstream.InputDStream
import org.apache.spark.streaming.kafka010.{ConsumerStrategies, KafkaUtils, LocationStrategies}
import org.apache.spark.streaming.{Seconds, StreamingContext}

import java.io.{File, FileWriter, PrintWriter}
import java.text.SimpleDateFormat
import java.util.Date
import scala.collection.mutable.ListBuffer

object SparkStreaming13_Req3_1 {
  def main(args: Array[String]): Unit = {
    val sparkConf = new SparkConf().setMaster("local[*]").setAppName("SparkStreaming")
    val ssc =new StreamingContext(sparkConf,Seconds(5))
    val kafkaPara: Map[String, Object] = Map[String, Object](
      ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG -> "hadoop102:9092,hadoop103:9092,hadoop104:9092",
      ConsumerConfig.GROUP_ID_CONFIG -> "atguigu",//消费者组ID
      "key.deserializer" -> "org.apache.kafka.common.serialization.StringDeserializer",
      "value.deserializer" -> "org.apache.kafka.common.serialization.StringDeserializer"
    )
    val kafkaDataDS: InputDStream[ConsumerRecord[String, String]] = KafkaUtils.createDirectStream[String, String](
      ssc,
      LocationStrategies.PreferConsistent, //位置策略，采集结点和计算结点如何做匹配？自己匹配：框架匹配
      ConsumerStrategies.Subscribe[String, String](Set("atguiguNew"), kafkaPara) //消费者策略,主题：atguiguNew
    )
    val adClickData = kafkaDataDS.map {
      kafkaData => {
        val data = kafkaData.value()
        val datas = data.split(" ")
        AdClickData(datas(0), datas(1), datas(2), datas(3), datas(4))
      }
    }
    //最近一分钟，每十秒计算一次
    val reduceDS = adClickData.map(
      data => {
        val ts = data.ts.toLong
        val newTS = ts / 10000 * 10000
        (newTS, 1)
      }
    ).reduceByKeyAndWindow((x:Int,y:Int)=>{x+y}, Seconds(60), Seconds(5))
//    reduceDS.print()
    reduceDS.foreachRDD(
      rdd =>{
        val list =ListBuffer[String]()
        val datas = rdd.sortByKey(true).collect()
        datas.foreach{
          case (time,cnt) =>{
            val timeString = new SimpleDateFormat("mm:ss").format(new Date(time.toLong))
            list.append(s"""{"xtime":"${timeString}", "yval":"${cnt}"}""")
          }
        }
        //输出文件
        val out =new PrintWriter(new FileWriter(new File("D:\\bigdataspark\\data\\adclick\\adclick.json")))
        out.println("["+list.mkString(",")+"]")
        out.flush()
        out.close()
      }
    )
    ssc.start()
    ssc.awaitTermination()
  }
  //广告点击数据样例类
  case class AdClickData(ts: String, area: String, city: String, user: String, ad: String)
}
